const axios = require("axios");
const cheerio = require("cheerio");
const { URL } = require("url");

async function crawlFullContent(url) {
  try {
    // 配置带超时和重试的请求
    const response = await axios.get(url, {
      timeout: 15000,
      headers: {
        "User-Agent":
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36",
        "Accept-Language": "en-US,en;q=0.9"
      },
      // 自动重定向
      maxRedirects: 5
    });

    const $ = cheerio.load(response.data);
    const baseUrl = response.request.res.responseUrl || url;

    // 结构化数据容器
    const result = {
      title: $("head > title").text() || "无标题",
      meta: {},
      links: new Set(),
      images: new Set(),
      scripts: new Set(),
      allVisibleText: "",
      rawHtml: response.data
    };

    // 提取元标签
    $("meta").each((_, element) => {
      const $meta = $(element);
      const name = $meta.attr("name") || $meta.attr("property");
      if (name) {
        result.meta[name] = $meta.attr("content") || "";
      }
    });

    // 提取所有链接并转换绝对路径
    $("a[href]").each((_, element) => {
      const href = $(element).attr("href");
      try {
        const absUrl = new URL(href, baseUrl).href;
        result.links.add(absUrl);
      } catch (e) {
        // 忽略无效链接
      }
    });

    // 提取图片和脚本
    $("img[src]").each((_, element) => {
      const src = $(element).attr("src");
      try {
        result.images.add(new URL(src, baseUrl).href);
      } catch {}
    });
    $("script[src]").each((_, element) => {
      const src = $(element).attr("src");
      try {
        result.scripts.add(new URL(src, baseUrl).href);
      } catch {}
    });

    // 提取所有可见文本（排除脚本/样式）
    $("body")
      .find("*")
      .not("script, style, link, meta, noscript")
      .each((_, element) => {
        const text = $(element).text().trim();
        if (text) {
          result.allVisibleText += text + "\n";
        }
      });

    // 返回结构化结果
    return {
      ...result,
      links: Array.from(result.links),
      images: Array.from(result.images),
      scripts: Array.from(result.scripts)
    };
  } catch (error) {
    console.error(`抓取失败: ${error.message}`);
    return null;
  }
}

// 使用示例
const targetUrl = "https://www.runoob.com/nodejs/nodejs-tutorial.html";
crawlFullContent(targetUrl).then((data) => {
  if (data) {
    console.log("抓取结果:", {
      title: data.title.slice(0, 50),
      linksCount: data.links.length,
      imagesCount: data.images.length,
      textPreview: data.allVisibleText.slice(0, 200) + "..."
    });
    // 可选：保存完整数据到JSON文件
    require("fs").writeFileSync("result.json", JSON.stringify(data, null, 2));
  }
});
